#!/bin/bash
. venv/bin/activate
cd spidertool
rm -f output.csv
rm -f commit_nums.csv
rm -f contributor_nums.csv
rm -f contributor_level.csv
rm -f github_tag.csv
rm -f scrapy.log

scrapy crawl github -s LOG_FILE=scrapy.log -a tag=$3 -a tid=$2 &
scrapy crawl openhub -s LOG_FILE=scrapy.log -a tag=$4 -a tid=$2 &
#scrapy crawl github_tag -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl baidu -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl bookask -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl csdn -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl soopat -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl xueshu -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl bing_academic -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl wanfang -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl cnblogs -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl iteye -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl itpub -s LOG_FILE=scrapy.log -a tag=$1 &
scrapy crawl nums_per_month -a tag=$4 -a tid=$2 &
scrapy crawl contributor_level -s LOG_FILE=scrapy.log -o contributor_level.csv -a tag=$4 &
spidertool/other_spiders/cnki.py $1 | tee -a scrapy.log &
spidertool/other_spiders/oschina.py $1 | tee -a scrapy.log &
spidertool/other_spiders/chinaunix.py $1 | tee -a scrapy.log &
spidertool/other_spiders/cnnvd.py $1 | tee -a scrapy.log &
spidertool/other_spiders/51cto.py $1 | tee -a scrapy.log &
spidertool/other_spiders/nvd.py $1 | tee -a scrapy.log &
spidertool/other_spiders/stackoverflow.py $1 | tee -a scrapy.log &
wait
./handleCSV.py | tee -a scrapy.log
./sortCSV.py contributor_nums.csv | tee -a scrapy.log
./sortCSV.py commit_nums.csv | tee -a scrapy.log
./sort_github_tag_CSV.py | tee -a scrapy.log
